{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db5f6a0e-b1c6-4214-95d4-79ccc9fa5b89",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import random\n",
    "from torch.utils.data import Dataset\n",
    "from pycocotools.coco import COCO\n",
    "from pycocotools import mask as maskUtils\n",
    "from lvis import LVIS\n",
    "from tqdm import tqdm\n",
    "from lvis import LVISEval, LVISResults\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14c254d1-42ca-4542-9406-e9f38b6ef553",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Begin Evaluation!\")\n",
    "lvis_ann = LVIS('./data/lvis/annotations/lvis_v1_val.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4268f01c-b71a-4952-bdc1-77d1b795e449",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('prediction_Qwen2_vl_7B_lvis_ov.json', 'r', encoding='utf-8') as f:\n",
    "    lvis_results =  json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c8da9d9-8187-4dab-b367-8f1096f38d06",
   "metadata": {},
   "outputs": [],
   "source": [
    "lvis_results = LVISResults(lvis_ann, lvis_results, max_dets=300)\n",
    "lvis_eval = LVISEval(lvis_ann, lvis_results, \"bbox\")\n",
    "lvis_eval.run()\n",
    "lvis_eval.print_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fec2dff7-349e-4ab3-a6de-d4cf3902f9cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_cate = ['horse_buggy', 'die', 'kitchen_table', 'omelet', 'papaya', 'stepladder']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd400d27-d458-491a-84da-165aaae545bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_cate = ['casserole', 'die', 'egg_roll', 'futon', 'garbage', 'handsaw', 'hippopotamus', 'kitchen_table', 'mallet', 'omelet', 'shot_glass', 'stepladder', 'sugar_bowl']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb1a4cb3-a4c4-4c21-8700-53a2768c2ef7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mAP and AP for selected_cate\n",
    "precisions = lvis_eval.eval['precision']\n",
    "cats = lvis_ann.load_cats(lvis_ann.get_cat_ids())\n",
    "cats_names = [item['name'] for item in cats]\n",
    "per_class_maps = []\n",
    "per_class_results = []\n",
    "for idx, name in enumerate(cats_names):\n",
    "    precision = precisions[:, :, idx, :]\n",
    "    precision = precision[precision > -1]\n",
    "    ap = np.mean(precision) if precision.size else float(\"nan\")\n",
    "    per_class_maps.append(float(ap * 100))\n",
    "    per_class_results.append({name:float(ap * 100)})\n",
    "\n",
    "ap_count = 0\n",
    "ap_sum = 0\n",
    "for item in per_class_results:\n",
    "    for k,v in item.items():\n",
    "        if k in selected_cate:\n",
    "            print(item)\n",
    "            ap_sum += v\n",
    "            ap_count += 1\n",
    "print({'mAP':ap_sum/ap_count})"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
